﻿Option Explicit On

Public Class frmKomcity
    Private cat As Object
    Private params As Object
    Private rubric As Integer
    Private rayon(0 To 1) As Integer
    Private is_run As Boolean
    Private Const SITE_URL As String = "http://www.komcity.ru/"
    Private adverts() As AdvertType
    Private cur_adv As Integer
    Public Const ADV_NAME As String = "komcity"
    Public file_name As String
    Public catalog_name As String
    Private h1 As String
    Private wbc As New System.Net.WebClient
    Private login As String
    Private session As String
    'Private IE As SHDocVw.InternetExplorer = New SHDocVw.InternetExplorer
    'Private webc As New clsWebWithCookies


    Private Sub frmKomcity_Load(ByVal sender As Object, ByVal e As System.EventArgs) Handles Me.Load
        'Dim response As String

        'Dim PostData As String = "login=" & login & "&password=" & login & "&from=/profile/"
        'wbc.Encoding = System.Text.Encoding.GetEncoding("windows-1251")
        'wbc.Headers.Add(Net.HttpRequestHeader.Cookie, "test=test1")
        'response = wbc.UploadString(SITE_URL & "authorization/", PostData)
        'SaveContent(response, "response.html")
        'For i As Integer = 0 To wbc.ResponseHeaders.Count - 1
        '    MsgBox(wbc.ResponseHeaders(i))
        'Next


        'Dim request As System.Net.HttpWebRequest
        'Dim stream As System.IO.Stream
        'Dim reader As System.IO.StreamReader
        'Dim str As String
        'request = DirectCast(System.Net.WebRequest.Create(SITE_URL), System.Net.HttpWebRequest)
        'stream = request.GetResponse().GetResponseStream()
        'reader = New System.IO.StreamReader(stream)
        'str = reader.ReadToEnd()
        'SaveContent(str, "response.html")
        'request = DirectCast(System.Net.WebRequest.Create(SITE_URL & "authorization/"), System.Net.HttpWebRequest)
        'request.Method = "POST"
        'request.ContentType = "application/x-www-form-urlencoded"
        'request.ContentLength = PostData.Length
        'Dim writer As New System.IO.StreamWriter(request.GetRequestStream(), System.Text.Encoding.GetEncoding("windows-1251"))
        'writer.Write(postData)
        'writer.Close()
        'request.GetResponse().GetResponseStream()
        'reader = New System.IO.StreamReader(stream)
        'str = reader.ReadToEnd()
        'SaveContent(str, "response.html")


        'Dim request As System.Net.WebRequest = System.Net.WebRequest.Create(SITE_URL & "authorization/")
        'request.Method = "post"
        'request.Headers.Add()
        'Dim response As System.Net.WebResponse = request.GetResponse()
        'Dim reader As System.IO.StreamReader = New System.IO.StreamReader(response.GetResponseStream())
        'Dim str As String = reader.ReadToEnd
        'Exit Sub

        Dim cent As String = "Центральный", dzem As String = "Дзёмги", co_lo As String = "ЦО и ЛО", _
            prigor As String = "Пригород", rayon As String = "Район", drug_np As String = "Другие н/п", _
            met As String = "Метал. ", kap As String = "Кап. "
        cat = {
            {0, "Комнаты", 0}, _
            {0, "Малосемейки", 1}, _
            {0, "1-комнатные", 2}, _
            {0, "2-комнатные", 3}, _
            {0, "3-комнатные", 4}, _
            {0, "4-комнатные", 5}, _
            {0, "5-комнатные", 6}, _
            {0, "Дома, коттеджи", 7}, _
            {0, "Дачи, участки", 8}, _
            {0, "Погреба", 9}, _
            {0, "Гаражи", 10}, _
            {0, "Нежилые помещения", 11}
        }
        params = { _
            {0, co_lo, 230, 231, "НашГород-Комнаты", "к"}, {0, prigor, 265, 0, "", "к"}, {0, rayon, 266, 0, "", "к"}, {0, drug_np, 267, 0, "", "к"}, _
            {1, co_lo, 232, 234, "НашГород-Малосемейки", "м"}, {1, prigor, 269, 0, "", "м"}, {1, rayon, 270, 0, "", "м"}, {1, drug_np, 271, 0, "", "м"}, _
            {2, co_lo, 235, 236, "НашГород-1", "1"}, {2, prigor, 273, 0, "", "1"}, {2, rayon, 274, 0, "", "1"}, {2, drug_np, 275, 0, "", "1"}, _
            {3, cent, 237, 0, "НашГород-2 ЦО", "2"}, {3, dzem, 238, 0, "НашГород-2 ЛО", "2"}, {3, prigor, 277, 0, "", "2"}, {3, rayon, 278, 0, "", "2"}, {3, drug_np, 279, 0, "", "2"}, _
            {4, cent, 239, 0, "НашГород-3 ЦО", "3"}, {4, dzem, 240, 0, "НашГород-3 ЛО", "3"}, {4, prigor, 281, 0, "", "3"}, {4, rayon, 282, 0, "", "3"}, {4, drug_np, 283, 0, "", "3"}, _
            {5, co_lo, 241, 242, "НашГород-4", "4"}, {5, prigor, 285, 0, "", "4"}, {5, rayon, 286, 0, "", "4"}, {5, drug_np, 287, 0, "", "4"}, _
            {6, co_lo, 243, 244, "НашГород-5", "5"}, {6, prigor, 289, 0, "", "5"}, {6, rayon, 290, 0, "", "5"}, {6, drug_np, 291, 0, "", "5"}, _
            {7, "", 11, 0, "", ""}, {8, "", 12, 0, "", ""}, {9, "", 13, 0, "", ""}, _
            {10, met & cent, 334, 0, "", ""}, {10, met & dzem, 335, 0, "", ""}, {10, met & "Без места", 338, 0, "", ""}, {10, met & prigor, 340, 0, "", ""}, _
            {10, kap & cent, 336, 0, "", ""}, {10, kap & dzem, 337, 0, "", ""}, {10, kap & drug_np, 339, 0, "", ""}, {10, kap & prigor, 341, 0, "", ""}, _
            {11, cent, 245, 0, "", ""}, {11, dzem, 246, 0, "", ""}, {11, prigor, 293, 0, "", ""}, {11, rayon, 294, 0, "", ""}, {11, drug_np, 295, 0, "", ""}
        }
        cboCatalog.SelectedIndex = 0
        web.ScriptErrorsSuppressed = True
        wbc.Encoding = System.Text.Encoding.Default 'UTF8
        lstConsole.Left = 0
        lstConsole.Dock = DockStyle.Fill
        'IE.Visible = True
    End Sub

    Private Sub frmKomcity_Shown(ByVal sender As Object, ByVal e As System.EventArgs) Handles Me.Shown
        GetIPAdress()

        Me.Refresh()

        Try 'Получаем дату обновления сайта Наш город
            Dim response As String = wbc.DownloadString(SITE_URL)
            Dim match As Match = Regex.Match(response, "<b>из газеты.*?Наш.*?город.*?от.*?(\d\d\.\d\d\.\d\d\d\d)</b>")
            If match.Success = True Then
                lblNGUpdate.Text = lblNGUpdate.Text & match.Groups(1).Value
                btnRun.Enabled = True
            End If
        Catch ex As Net.WebException
            lblNGUpdate.Text = lblNGUpdate.Text & "Страница не получена."
        End Try

        'Подгружаем логин для komcity
        login = GetINIString("komcity", "login")
        If login = "" Then
            MsgBox("В настройках не указан login для сайта komcity." & vbCrLf & "Продолжение работы граббера не возможна.", MsgBoxStyle.Critical)
            Me.Close()
        End If
    End Sub



    Private Sub cboCatalog_SelectedIndexChanged(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles cboCatalog.SelectedIndexChanged
        Dim n As Integer
        'Отчистим списки рубрика и район
        cboRubric.Items.Clear()
        cboRayon.Items.Clear()
        'Заполняем список рубрика
        For n = 0 To UBound(cat)
            If cat(n, 0) = cboCatalog.SelectedIndex Then
                cboRubric.Items.Add(cat(n, 1))
            End If
        Next

        cboRubric.SelectedIndex = 0
    End Sub

    Private Sub cboRubric_SelectedIndexChanged(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles cboRubric.SelectedIndexChanged
        Dim n As Integer
        'Найдем id рубрики
        For n = 0 To UBound(cat)
            If cat(n, 0) = cboCatalog.SelectedIndex And cat(n, 1) = cboRubric.Text Then
                rubric = cat(n, 2)
                Exit For
            End If
        Next

        cboRayon.Items.Clear()
        For n = 0 To UBound(params)
            If params(n, 0) = rubric Then
                cboRayon.Items.Add(params(n, 1))
            End If
        Next

        cboRayon.SelectedIndex = 0
    End Sub

    Private Sub cboRayon_SelectedIndexChanged(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles cboRayon.SelectedIndexChanged
        Dim n As Integer
        'Найдем id района
        For n = 0 To UBound(params)
            If params(n, 0) = rubric And params(n, 1) = cboRayon.Text Then
                rayon(0) = params(n, 2)
                rayon(1) = params(n, 3)
                file_name = params(n, 4)
                catalog_name = params(n, 5)
                Exit For
            End If
        Next
    End Sub


    'Авторизация на сайте
    Private Function Authorization() As Boolean
        myConsole(lstConsole, "Авторизуемся на сайте под логином " & login)
        Try
            WebReadyState(web, SITE_URL)
            'web.Document.Encoding = "windows-1251"
            Dim form As System.Windows.Forms.HtmlElement
            Dim match As Match
            form = web.Document.Forms(0)
            If form.GetAttribute("action") = "/authorization/" Then
                web.Document.GetElementById("login").SetAttribute("value", login)
                web.Document.GetElementById("password").SetAttribute("value", login)
                form.InvokeMember("submit")
                Threading.Thread.Sleep(2000)
                Return ReadyStateComplete(web, 40000)
            Else
                myConsole(lstConsole, "Уже авторизованы")
                Return True
            End If
        Catch ex As Exception
            myConsole(lstConsole, "Ошибка кода при авторизации")
            myConsole(lstConsole, ex.Message)
            Return False
        End Try
    End Function


    Private Sub btnRun_Click(ByVal sender As Object, ByVal e As System.EventArgs) Handles btnRun.Click
        Dim response As String, match As Match
        Dim is_login As Boolean

        'Авторизуемся на сайте и получаем session id
        is_login = Authorization()
        If is_login = False Then
            myConsole(lstConsole, "Не получилось авторизоваться на сайте!!!")
            myConsole(lstConsole, "Попробуйте еще раз сделать запуск")
            lstConsole.Items.Add("")
            Exit Sub
        Else
            match = Regex.Match(web.Document.Cookie, "session=(.*?);")
            If match.Success = True Then
                session = match.Groups(1).Value
            Else
                myConsole(lstConsole, "В Cookies не найден параметр session!!!")
                Exit Sub
            End If
        End If

        btnRun.Enabled = False

        cur_adv = 0
        ReDim adverts(0)
        wbc.Encoding = System.Text.Encoding.UTF8
        For i As Integer = 0 To UBound(rayon)
            If rayon(i) = 0 Then Continue For

            'Получить страницу
            Try : response = wbc.DownloadString(SITE_URL & "board/main/getPubAdverts/?id=" & rayon(i))
            Catch ex As Net.WebException
                myConsole(lstConsole, "Страница не получена. " & ex.Message) : Exit Sub
            End Try

            'Получаем со страницы все объявления
            For Each match In Regex.Matches(response, "<div id=""AllText""><a.*?viewData\((\d{4,10})\);"">(.*?)</a>", RegexOptions.Singleline)
                ReDim Preserve adverts(cur_adv)
                adverts(cur_adv).text = match.Groups(2).Value.Trim()
                adverts(cur_adv).url = "board/main/GetContact/?id=" & match.Groups(1).Value
                adverts(cur_adv).advid = match.Groups(1).Value
                cur_adv = cur_adv + 1
            Next
        Next i

        'Запускаем граббер
        h1 = "Наш город - " & cboCatalog.Text & " " & cboRubric.Text & " " & cboRayon.Text
        Me.Text &= " - " & cboCatalog.Text & " " & cboRubric.Text & " " & cboRayon.Text
        cur_adv = 0
        txtCur.Text = 0
        txtMax.Text = adverts.Count
        myConsole(lstConsole, "Найдено " & adverts.Count & " объявлений")
        lstConsole.Items.Add("")
        btnPlayPause.Visible = True
        btnPlayPause.PerformClick()
    End Sub


    Private Sub btnPlayPause_Click(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles btnPlayPause.Click
        If is_run = False Then 'Сейчас пауза, нужно запустить
            btnPlayPause.Text = "Приостановить"
            timTick.Interval = 500
            timTick.Start()
        Else 'Сейчас запущено, нужно сделать паузу
            btnPlayPause.Text = "Запустить"
            timTick.Stop()
        End If
        is_run = Not is_run
    End Sub


    Private Sub lblLocalIP_Click(ByVal sender As Object, ByVal e As System.EventArgs) Handles lblLocalIP.Click
        GetIPAdress()
    End Sub


    Private Sub GetIPAdress()
        Dim response As String = "", s As String = "Ваш IP адрес: "
        lblLocalIP.Text = s & "..."
        Me.Refresh()
        Try
            response = wbc.DownloadString("http://advertsgrabber.esy.es/env.php")
            If Regex.IsMatch(response, "^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$") Then
                lblLocalIP.Text = s & response
            Else
                lblLocalIP.Text = s & "не получен"
            End If
        Catch ex As Net.WebException
            lblLocalIP.Text = s & ex.Message
        End Try
    End Sub



    Private Sub timTick_Tick(ByVal sender As Object, ByVal e As System.EventArgs) Handles timTick.Tick
        Dim response As String
        Dim trim_chars() As Char = {",", ".", " "}
        Dim flag As Boolean
        Dim tmpS As String, tmpS2 As String, tmpO As Object
        Dim n As Integer
        Dim match As Match

        timTick.Stop()

        If is_run = False Then Exit Sub

        txtCur.Text = cur_adv + 1
        myConsole(lstConsole, (cur_adv + 1) & " объявление загружается")

        With adverts(cur_adv)
            '.advid = "2099562"
            '.url = "board/main/GetContact/?id=" & .advid

            'Проверим если объявление старое, то не загружаем его
            If chkOnlyNew.Checked = True Then
                If DBAdverts_isOldAdvert(ADV_NAME, .advid) = True Then
                    myConsole(lstConsole, "---Это объявление старое")
                    .state = AdvState.old
                    GoTo next_advert
                End If
            End If

            'Получаем номер телефона
            Try : response = wbc.DownloadString(SITE_URL & .url & "&h=" & session)
            Catch ex As Net.WebException
                myConsole(lstConsole, "Страница не получена. " & ex.Message)
                btnPlayPause.PerformClick()
                Exit Sub
            End Try

            'Проверим правильно ли получен номер телефона
            If response.IndexOf("script") > -1 Then
                myConsole(lstConsole, "---Потеряна сессия авторизации")
                Exit Sub
            End If

            response = Regex.Replace(response, "[\. ]", "")
            response = Regex.Replace(response, "Обращаться.*Тел", "")
            response = Regex.Replace(response, "Тел:", "")
            response = Regex.Replace(response, "Тел", "")

check_phones:
            'Получаем массив телефонов
            .phone = getPhonesArray(response)
            'Проверяем каждый полученный телефон
            For n = 0 To UBound(.phone)
                tmpO = getBigPhone(.phone(n))
                'Если телефон неверный, то запрашиваем у юзера ручной ввод телефона
                If Equals(tmpO, False) = True Then
                    frmAdvertPhone.file_name = ""
                    frmAdvertPhone.txtPhone.Text = response
                    frmAdvertPhone.ShowDialog(Me)
                    response = Join(frmAdvertPhone.phones, ",")
                    frmAdvertPhone.Dispose()
                    GoTo check_phones
                End If
            Next

            'Получаем номер телефона, ищем его в риэлторах
            'GoTo skip_phone
            For n = 0 To UBound(.phone)
                If DBEstate_isEstate(.phone(n), True) = True Then
                    myConsole(lstConsole, "---Это риэлтор")
                    .state = AdvState.estateInDB
                    'Сохраняем объявление с риэлтором как старое
                    If DBAdverts_isOldAdvert(ADV_NAME, .advid) = False Then
                        DBAdverts_addOldAdvert(ADV_NAME, .advid)
                    End If
                    'Переходим к следующему объявлению
                    GoTo next_advert
                End If
            Next

skip_phone:
            .state = AdvState.normal
            .text = .text.Replace("  ", " ").Trim(trim_chars)
            .text = Regex.Replace(.text, "^Комнату", "").Trim(trim_chars)

            'Улица и номер
            flag = False : tmpS = "" : tmpS2 = "" : n = 0
            For Each match In Regex.Matches(.text, "([а-яА-Я0-9-\\/]+)")
                n = n + 1
                If n = 2 Or n = 3 Then
                    tmpS2 = match.Groups(1).Value.Trim()
                    If Regex.IsMatch(tmpS2, "^[0-9\\/]+$") Then
                        .street = tmpS.Trim()
                        .nomer = tmpS2
                        .text = .text.Substring(match.Index + match.Length).Trim(trim_chars)
                        Exit For
                    End If
                End If
                If n = 1 Or n = 2 Then
                    tmpS = tmpS & match.Groups(1).Value & " "
                End If
                If n > 3 Then
                    Exit For
                End If
            Next
            .AddContent(.text)

            .text = " " & .text

            'Этаж/этажность
            match = Regex.Match(.text, "([0-9]+)/([0-9]+) эт")
            If match.Success = True Then
                .floor = match.Groups(1).Value
                .floor_all = match.Groups(2).Value
            End If

            'Квадраты
            match = Regex.Match(.text, "([0-9\.,]*)/([0-9\.,]*)/([0-9\.,]*) ")
            If match.Success = True Then
                .kvm_all = match.Groups(1).Value
                .kvm_live = match.Groups(2).Value
                .kvm_kitchen = match.Groups(3).Value
            End If

            'Цена
            match = Regex.Match(.text, "([0-9 ]{3,}) т\.р")
            If match.Success = True Then
                .price = match.Groups(1).Value.Replace(" ", "")
            End If

            .text = .text.Trim()
next_advert:
            lstConsole.Items.Add("")
            If cur_adv = adverts.Count - 1 Then
                'Объявления кончились, выводим их
                web.Navigate("about:blank")
                ReadyStateComplete(web, 10000)
                Threading.Thread.Sleep(1000)
                web.Visible = True
                web.Left = 0
                web.Dock = DockStyle.Fill
                grbControl.Enabled = False
                lstConsole.Visible = False

                'Подготавливаем список перед выводом
                AdvertsCreateList(adverts, chkSaveNew.Checked, ADV_NAME)

                'Собираем HTML код и обновляем WebBrowser
                AdvertsShowList(web, adverts, h1, AddressOf SpanPhoneClick, ADV_NAME)
            Else
                'Смотрим следующее объявление
                If .state = AdvState.old Then 'Если это старая объява, то не ждем
                    timTick.Interval = 100
                Else 'Ждем 10 сек
                    timTick.Interval = 8000
                End If
                timTick.Start()
            End If
        End With
        cur_adv = cur_adv + 1
    End Sub



    Private Sub SpanPhoneClick(ByVal sender As Object, ByVal e As System.EventArgs)
        AdvertsSpanPhoneClick(web, adverts, h1, AddressOf SpanPhoneClick, ADV_NAME)
    End Sub



    Public Function getAdverts() As Object
        Return adverts
    End Function
End Class